package com.collection.service.impl;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.xwork.StringUtils;
import org.apache.log4j.Logger;
import org.htmlcleaner.CleanerProperties;
import org.htmlcleaner.HtmlCleaner;
import org.htmlcleaner.TagNode;
import org.htmlcleaner.XPatherException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.Assert;

import com.collection.dao.CollectionTaskDao;
import com.collection.entity.BaseStrategy;
import com.collection.entity.CollectionModelToPublishModel;
import com.collection.entity.CollectionTask;
import com.collection.entity.ContentStrategy;
import com.collection.entity.FilterTag;
import com.collection.entity.PublishConfig;
import com.collection.entity.StrategyValue;
import com.collection.entity.UrlParam;
import com.collection.entity.WebsiteStrategy;
import com.collection.entity.CollectionModelToPublishModel.DataType;
import com.collection.entity.CollectionModelToPublishModel.ModelType;
import com.collection.entity.PublishConfig.DB;
import com.collection.service.CollectionTaskService;
import com.collection.util.Constants;
import com.collection.util.DBUtil;
import com.collection.util.ParseEncoding;

@Service
public class CollectionTaskServiceImpl implements CollectionTaskService {

	private static Logger logger = Logger.getLogger(CollectionTaskServiceImpl.class);

	@Autowired
	private CollectionTaskDao collectionTaskDao;

	private String tableName;
	
	private boolean isTest;

	/*
	 * (non-Javadoc)
	 * 
	 * @see
	 * com.collection.service.CollectionTaskService#createCollectionTask(com
	 * .collection.entity.CollectionTask)
	 */
	@Override
	public void createCollectionTask(CollectionTask collectionTask) {
		if (collectionTask != null) {
			collectionTaskDao.save(collectionTask);
		}
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see
	 * com.collection.service.CollectionTaskService#deleteCollectionTask(com
	 * .collection.entity.CollectionTask)
	 */
	@Override
	public void deleteCollectionTask(CollectionTask collectionTask) {
		if (collectionTask != null) {
			collectionTaskDao.delete(collectionTask);
		}
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see
	 * com.collection.service.CollectionTaskService#deleteCollectionTask(java
	 * .lang.Integer)
	 */
	@Override
	public void deleteCollectionTask(Integer id) {
		if (id != null) {
			CollectionTask collectionTask = getCollectionTaskById(id);
			if (collectionTask != null) {
				collectionTaskDao.delete(collectionTask);
			}
		}
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see
	 * com.collection.service.CollectionTaskService#findAllCollectionTasks()
	 */
	@Override
	public List<CollectionTask> findAllCollectionTasks() {
		return collectionTaskDao.queryByHQL("from CollectionTask t order by t.updateDate desc");
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see
	 * com.collection.service.CollectionTaskService#getCollectionTaskById(java
	 * .lang.Integer)
	 */
	@Override
	public CollectionTask getCollectionTaskById(Integer id) {
		if (id != null) {
			return collectionTaskDao.get(CollectionTask.class, id);
		}
		return null;
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see
	 * com.collection.service.CollectionTaskService#modifyCollectionTask(com
	 * .collection.entity.CollectionTask)
	 */
	@Override
	public void modifyCollectionTask(CollectionTask collectionTask) {
		if (collectionTask != null) {
			// Integer id = collectionTask.getId();
			// String taskName = collectionTask.getTaskName();
			// String alias = collectionTask.getTaskAlias();
			//			
			// collectionTask = getCollectionTaskById(id);
			//			
			// collectionTask.setTaskName(taskName);
			// collectionTask.setTaskAlias(alias);
			collectionTask.setUpdateDate(new Date());
			collectionTaskDao.update(collectionTask);
		}
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see
	 * com.collection.service.CollectionTaskService#aliasExsit(java.lang.String)
	 */
	public boolean taskAliasExist(String alias) {
		List<CollectionTask> cts = collectionTaskDao.queryByHQL("from CollectionTask ct where ct.taskAlias=?",
				new Object[] { alias });
		if (cts != null && cts.size() > 0) {
			return true;
		} else {
			return false;
		}
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see
	 * com.collection.service.CollectionTaskService#collectionContent(com.collection
	 * .entity.ContentStrategy, java.lang.String)
	 */
	@Override
	public List<Map<String, String>> collectionContent(ContentStrategy contentStrategy) {
		List<Map<String, String>> dataMaps = new ArrayList<Map<String, String>>();
		if (contentStrategy != null) {
			List<BaseStrategy> baseStrategies = contentStrategy.getBaseStrategies();
			String charCode = contentStrategy.getCharCode();

			if (baseStrategies != null && baseStrategies.size() > 0) {

				/*
				 * 需要采集的网址总数
				 */
				int totalUrls = countHasNoCollectionUrls(tableName);

				if (totalUrls > 0) {

					List<String> urls = null;

					int pageSize = 1000;

					int totalPage = (totalUrls + pageSize - 1) / pageSize + 1;
					
					/*
					 * 分页采集所有网址
					 */
					for (int pageNo = 1; pageNo <= totalPage; pageNo++) {
						urls = findHasNoCollectionUrls(tableName, (pageNo - 1) * pageSize, pageSize);

						if (urls != null && urls.size() > 0) {
							Map<String, String> dataMap = null;
							for (String url : urls) {
								dataMap = new HashMap<String, String>();
								for (int i = 0; i < baseStrategies.size(); i++) {
									BaseStrategy baseStrategy = baseStrategies.get(i);
									String fieldName = baseStrategy.getFieldName();
									List<String> collectionDatas = collectionData(url, charCode, baseStrategy);

									if (collectionDatas != null && collectionDatas.size() > 0) {
										dataMap.put(fieldName, collectionDatas.get(0));
									}

									List<String> contents = new ArrayList<String>();

									/*
									 * 采集url中的分页内容
									 */
									collectionPageContent(url, charCode, baseStrategy, contents);

									if (contents.size() > 0) {
										String content = dataMap.get(fieldName);
										for (String c : contents) {
											content += "<!-- pagebreak -->" + c;   //加入内容分页标志
										}
										dataMap.put(fieldName, content);
									}
									dataMaps.add(dataMap);
									
									if(!isTest) {
										/*
										 * 导入采集的内容
										 */
										importPageContent(dataMap, url, tableName);
									}
								}
							}
						}
					}
				}
			}
		}
		return dataMaps;
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see
	 * com.collection.service.CollectionTaskService#collectionWebsite(com.collection
	 * .entity.WebsiteStrategy)
	 */
	@Override
	public List<String> collectionWebsite(WebsiteStrategy websiteStrategy) {
		List<String> collectionDatas = new ArrayList<String>();
		if (websiteStrategy != null) {
			String url = websiteStrategy.getUrl();

			boolean autoPage = websiteStrategy.isAutoPage();
			List<String> topUrls = new ArrayList<String>();
			String charCode = websiteStrategy.getCharCode();
			/*
			 * 获取所有一级页面的URL
			 */
			if (autoPage) {
				List<UrlParam> urlParams = websiteStrategy.getUrlParams();
				createUrl(urlParams, url, 0, topUrls);
			} else {
				topUrls = Arrays.asList(url.split("\n\r"));
			}

			List<BaseStrategy> baseStrategies = websiteStrategy.getBaseStrategies();
			// if(baseStrategies != null && baseStrategies.size() > 0) {
			// BaseStrategy baseStrategy = baseStrategies.get(0);
			// for (String tempUrl : topUrls) {
			// /*
			// * 采集所有二级页面的URL
			// */
			// List<String> collectionUrls = collectionData(tempUrl,
			// charCode, baseStrategy);
			// for (String collectionUrl : collectionUrls) {
			// collectionDatas.add(urlHandler(tempUrl, collectionUrl));
			// }
			// }
			// }

			List<BaseStrategy> collectionBaseStrategies = new ArrayList<BaseStrategy>();

			if (baseStrategies != null) {
				for (BaseStrategy bs : baseStrategies) {
					collectionBaseStrategies.add(bs);
				}
			}

			collectionDatas = collectionUrls(charCode, collectionBaseStrategies, topUrls);
		}
		return collectionDatas;
	}

	private List<String> collectionUrls(String charCode, List<BaseStrategy> baseStrategies, List<String> nextUrls) {
		if (baseStrategies != null) {
			int strategiesLength = baseStrategies.size();

			/*
			 * 当为最后一级网址采集策略时返回采集的网址信息
			 */
			if (strategiesLength == 0) {
				return nextUrls;
			}

			BaseStrategy baseStrategy = baseStrategies.get(0);

			/*
			 * 采集所有下级页面的URL
			 */
			List<String> urls = new ArrayList<String>();
			if (nextUrls.size() > 0) {
				for (String nextUrl : nextUrls) {

					/*
					 * 采集nextUrl的下一级URL
					 */
					collectionNextUrls(nextUrl, charCode, baseStrategy, urls);

					collectionPageContent(nextUrl, charCode, baseStrategy, urls);

					if (strategiesLength == 1 && !isTest) {
						importPageUrl(urls, tableName);
					}
				}
			}
			baseStrategies.remove(baseStrategy);
			return collectionUrls(charCode, baseStrategies, urls);
		}
		return nextUrls;
	}

	/**
	 * 采集分页内容
	 * 
	 * @param nextUrl
	 * @param charCode
	 * @param baseStrategy
	 * @param collectionContents
	 */
	private void collectionPageContent(String nextUrl, String charCode, BaseStrategy baseStrategy,
			List<String> collectionContents) {
		String pageStartTag = baseStrategy.getPageStartTag();
		String pageStartEnd = baseStrategy.getPageEndTag();
		Integer pageNums = baseStrategy.getPageNums();

		if (StringUtils.isNotBlank(pageStartTag) && StringUtils.isNotBlank(pageStartEnd)) {

			/*
			 * 自动分页URL
			 */
			String nextPageReg = autoGetPageUrl(nextUrl, charCode, pageStartTag, pageStartEnd);

			if (StringUtils.isNotBlank(nextPageReg)) {
				if (nextPageReg.contains("{*}")) {
					if (pageNums != null && pageNums > 0) {
						for (int j = 2; j <= pageNums; j++) {

							String pageUrl = urlHandler(nextUrl, nextPageReg.replace("{*}", j + ""));

							/*
							 * 采集pageUrl的下一级URL，如果采集不成功。说明接下来的分页都没有数据可采集
							 */
							if (!collectionNextUrls(pageUrl, charCode, baseStrategy, collectionContents)) {
								break;
							}
						}
					} else {
						int index = 2;
						while (true) {
							String pageUrl = urlHandler(nextUrl, nextPageReg.replace("{*}", index + ""));

							/*
							 * 采集pageUrl的下一级URL，如果采集不成功。说明接下来的分页都没有数据可采集
							 */
							if (!collectionNextUrls(pageUrl, charCode, baseStrategy, collectionContents)) {
								break;
							}
							index++;
						}
					}
				} else {
					String pageUrl = urlHandler(nextUrl, nextPageReg);

					/*
					 * 采集pageUrl的下一级URL
					 */
					collectionNextUrls(pageUrl, charCode, baseStrategy, collectionContents);
				}
			}
		}
	}

	/*
	 * 采集当前url的内容
	 */
	private boolean collectionNextUrls(String currentUrl, String charCode, BaseStrategy baseStrategy,
			List<String> collectionContents) {
		List<String> collectionContent = collectionData(currentUrl, charCode, baseStrategy);

		if (collectionContent != null && collectionContent.size() > 0) {
			System.out.println("---" + currentUrl);
			for (String collectionUrl : collectionContent) {
				collectionUrl = urlHandler(currentUrl, collectionUrl);
				collectionContents.add(collectionUrl);
				System.out.println("---------" + collectionUrl);
			}
			return true;
		} else {
			return false;
		}
	}

	public String autoGetPageUrl(String url, String charCode, String pageStartTag, String pageStartEnd) {

		/*
		 * 获取url内容
		 */
		String content = getHtmlContent(url, charCode);
		int pos = content.indexOf(pageStartTag);
		int nextPos = content.indexOf(pageStartEnd, pos);
		if (pos != -1 && nextPos != -1 && nextPos > pos) {
			content = content.substring(pos + 1, nextPos);

			if (StringUtils.isNotBlank(content)) {
				HtmlCleaner cleaner = new HtmlCleaner();

				TagNode[] tagNodes = cleaner.clean(content).getElementsByName("a", true);

				if (tagNodes != null && tagNodes.length > 1) {
					String prePageUrl = null;
					String nextPageUrl = null;

					for (int i = 0; i < tagNodes.length - 1; i++) {
						prePageUrl = tagNodes[i].getAttributeByName("href");
						nextPageUrl = tagNodes[i + 1].getAttributeByName("href");

						if (StringUtils.isNotBlank(prePageUrl) && StringUtils.isNotBlank(nextPageUrl)) {
							break;
						}
					}

					if (StringUtils.isNotBlank(prePageUrl) && StringUtils.isBlank(nextPageUrl)) {
						return prePageUrl;
					} else if (StringUtils.isBlank(prePageUrl) && StringUtils.isNotBlank(nextPageUrl)) {
						return nextPageUrl;
					} else if (StringUtils.isBlank(prePageUrl) && StringUtils.isBlank(nextPageUrl)) {
						return null;
					} else if (nextPageUrl.equals(prePageUrl)) {
						return nextPageUrl;
					}

					Pattern p = Pattern.compile("\\d+");

					Matcher pM = p.matcher(prePageUrl);

					Matcher nM = p.matcher(nextPageUrl);

					List<String> puS = new ArrayList<String>();

					List<String> nuS = new ArrayList<String>();

					while (pM.find()) {
						puS.add(pM.group());
					}

					while (nM.find()) {
						nuS.add(nM.group());
					}

					if (puS.size() == nuS.size()) {
						int numpStartIndex = -1;
						int numnStartIndex = -1;
						for (int i = 0; i < puS.size(); i++) {
							String numpS = puS.get(i);
							String numnS = nuS.get(i);
							System.out.println(numpS + "--------" + numnS);

							/*
							 * 必须从上一个数字结束开始
							 */
							int numpIndex = prePageUrl.indexOf(numpS, numpStartIndex);
							int numnIndex = nextPageUrl.indexOf(numnS, numnStartIndex);

							numpStartIndex = numpIndex + numpS.length();
							numnStartIndex = numnIndex + numnS.length();
							if ((numpIndex == numnIndex) && !numpS.equals(numnS)) {
								return prePageUrl.replace(numpS, "{*}");
							}
						}
					}
				}
			}
		}
		return null;
	}

	private String urlHandler(String urlPath, String url) {
		if (url != null && !url.startsWith("http://")) {
			String httpUrl = "";
			if (url.startsWith("/")) {
				httpUrl = "http://" + getHost(urlPath);
			} else {
				if (urlPath.endsWith("/")) {
					httpUrl = urlPath;
				} else {
					httpUrl = urlPath.substring(0, urlPath.lastIndexOf("/") + 1);
				}

			}
			url = httpUrl + url;
		}
		return url;
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see
	 * com.collection.service.CollectionTaskService#collectionData(java.lang
	 * .String, java.lang.String, java.lang.String,
	 * com.collection.entity.BaseStrategy)
	 */
	public List<String> collectionData(String url, String charCode, BaseStrategy baseStrategy) {
		List<String> collectionDatas = new ArrayList<String>();
		BaseStrategy.Strategy strategy = baseStrategy.getStrategy();
		StrategyValue strategyValue = baseStrategy.getStrategyValue();
		List<FilterTag> filterTags = baseStrategy.getFilterTag();
		Boolean download = baseStrategy.getDownload();
		String downloadDir = baseStrategy.getDownloadDir();
		/*
		 * 如果字符编码没有指定，需要自动获取字符编码
		 */
		if (StringUtils.isBlank(charCode)) {
			ParseEncoding parse = new ParseEncoding();
			charCode = parse.getEncoding(url);
		}

		if (BaseStrategy.Strategy.TAG.equals(strategy)) {
			/*
			 * 获取html内容
			 */
			String content = getHtmlContent(url, charCode);
			String startTag = strategyValue.getStartTag();
			String endTag = strategyValue.getEndTag();
			int pos = content.indexOf(startTag);
			int nextPos = content.indexOf(endTag, pos);
			while (pos != -1 && nextPos != -1) {

				/*
				 * 将匹配到的数据加入采集列表中
				 */
				collectionDatas.add(dataHandler(url, content.substring(pos + startTag.length(), nextPos), filterTags,
						download, downloadDir));

				/*
				 * 从已经匹配的位置继续找下一个匹配的数据
				 */
				pos = content.indexOf(startTag, nextPos);
				nextPos = content.indexOf(endTag, pos);
			}

		} else if (BaseStrategy.Strategy.REGULAR.equals(strategy)) {
			/*
			 * 获取html内容
			 */
			String content = getHtmlContent(url, charCode);
			Pattern p = Pattern.compile(strategyValue.getRegular());
			Matcher matcher = p.matcher(content);

			while (matcher.find()) {
				collectionDatas.add(dataHandler(url, matcher.group(), filterTags, download, downloadDir));
			}
		} else if (BaseStrategy.Strategy.XPATH.equals(strategy)) {
			HtmlCleaner cleaner = new HtmlCleaner();
			CleanerProperties props = cleaner.getProperties();
			props.setOmitComments(true);
			props.setAdvancedXmlEscape(false);
			props.setRecognizeUnicodeChars(false);
			props.setTranslateSpecialEntities(false);

			TagNode node = null;
			try {
				node = cleaner.clean(new URL(url), charCode);

				// 按tag取.
				Object[] ns = null;
				String xpath = strategyValue.getXpath();
				String xpathMode = strategyValue.getXpathMode();
				if (StringUtils.isNotBlank(xpathMode)) {
					String[] mode = xpathMode.split(":");

					if ("attr".equals(mode[0])) {
						ns = node.evaluateXPath(xpath);
						for (int i = 0; i < ns.length; i++) {
							collectionDatas.add(dataHandler(url, ((TagNode) ns[i]).getAttributeByName(mode[1]),
									filterTags, download, downloadDir));
						}
					} else if ("tag".equals(mode[0])) {
						ns = node.getElementsByName(mode[1], true); // 标题
						for (int i = 0; i < ns.length; i++) {
							StringBuffer text = ((TagNode) ns[i]).getText();
							if (text != null) {
								collectionDatas
										.add(dataHandler(url, text.toString(), filterTags, download, downloadDir));
							}
						}
					}
				} else {
					ns = node.evaluateXPath(xpath);
					for (int i = 0; i < ns.length; i++) {
						// StringBuffer text = ((TagNode) ns[i]).getText();
						String in = cleaner.getInnerHtml((TagNode) ns[i]);
						System.out.println(in);
						// System.out.println(text);
						collectionDatas.add(dataHandler(url, in, filterTags, download, downloadDir));
					}
				}
			} catch (MalformedURLException e) {
				e.printStackTrace();
			} catch (IOException e) {
				e.printStackTrace();
			} catch (XPatherException e) {
				e.printStackTrace();
			}
		}
		return collectionDatas;
	}

	private String dataHandler(String urlPath, String data, List<FilterTag> filterTags, Boolean download,
			String downloadDir) {
		if (filterTags != null && filterTags.size() > 0) {
			for (FilterTag filterTag : filterTags) {
				String tag = filterTag.getTagName();
				data = data.replaceAll("<(" + tag + "|" + tag.toUpperCase() + ")[^>]*>", "").replaceAll(
						"</(" + tag + "|" + tag.toUpperCase() + ")>", "");
			}
		}
		if (download != null && download) {
			data = downloadFile(data, urlPath, downloadDir);
		}
		return data;
	}

	private String downloadFile(String data, String urlPath, String downloadDir) {

		System.setProperty("sun.net.client.defaultConnectTimeout", "60000");
		System.setProperty("sun.net.client.defaultReadTimeout", "60000");

		if (StringUtils.isBlank(downloadDir)) {
			downloadDir = "download";
		}
		String datePath = new SimpleDateFormat("yyyyMM").format(new Date());
		Pattern patt = Pattern.compile("(src|SRC)=\"(\\S)+\"");
		Matcher m = null;

		ParseEncoding parse;

		parse = new ParseEncoding();

		System.out.println("desc编码：" + parse.getEncoding(data.getBytes()));
		m = patt.matcher(data);

		while (m.find()) {
			String imageSrcObj = m.group();

			int pos = imageSrcObj.lastIndexOf("\"");

			if (pos == -1) {
				data = data.replaceAll("<(img|IMG)[^>]*" + imageSrcObj + "[^>]* />", "");
				continue;
			}

			String imageSrc = imageSrcObj.substring(5, pos);

			pos = imageSrc.lastIndexOf(".");
			if (pos == -1) {
				data = data.replaceAll("<(img|IMG)[^>]*" + imageSrcObj + "[^>]* />", "");
				continue;
			}

			if (StringUtils.isNotBlank(imageSrc)) {
				String httpUrl = urlHandler(urlPath, imageSrc);
				InputStream is = null;
				OutputStream os = null;
				try {
					// 创建本地文件和目录
					String ext = imageSrc.substring(pos);
					if (Constants.imageExts.contains(ext.toLowerCase())) {
						String fileName = UUID.randomUUID().toString();
						String descFile = downloadDir + "/" + datePath + "/" + fileName + ext;
						File file = new File(descFile);
						if (!file.exists() || file.length() <= 0) {
							Assert.notNull(file, "文件不能为空");
							File parentFile = file.getParentFile();
							if (!parentFile.exists()) {
								parentFile.mkdirs();
							}
							if (!file.exists()) {
								file.createNewFile();
							}
							// 抓取目标图片到本地
							URL url = new URL(httpUrl);
							is = url.openStream();
							os = new FileOutputStream(file);
							int byteRead = 0;
							byte[] buffer = new byte[8192];
							while ((byteRead = is.read(buffer, 0, 8192)) != -1) {
								os.write(buffer, 0, byteRead);
							}
							data = data.replace(imageSrc, descFile);
						}
					}
				} catch (Exception e) {
					logger.error("无法下载图片" + httpUrl, e);
				} finally {
					IOUtils.closeQuietly(is);
					IOUtils.closeQuietly(os);
				}
			}
		}
		return data;
	}

	private String copySRCFileToDescFile(String data, String appRootPath, String descFileDir) {

		if (StringUtils.isNotBlank(data)) {
			String datePath = new SimpleDateFormat("yyyyMM").format(new Date());
			Pattern patt = Pattern.compile("(src|SRC)=\"(\\S)+\"");
			Matcher m = null;

			ParseEncoding parse;

			parse = new ParseEncoding();

			System.out.println("desc编码：" + parse.getEncoding(data.getBytes()));
			m = patt.matcher(data);

			String fileName = "";
			while (m.find()) {
				String imageSrcObj = m.group();

				int pos = imageSrcObj.lastIndexOf("\"");

				if (pos == -1) {
					data = data.replaceAll("<(img|IMG)[^>]*" + imageSrcObj + "[^>]* />", "");
					continue;
				}

				String imageSrc = imageSrcObj.substring(5, pos);

				pos = imageSrc.lastIndexOf("/");
				if (pos != -1) {
					fileName = imageSrc.substring(pos + 1);
				}

				if (StringUtils.isNotBlank(imageSrc)) {
					FileInputStream fis = null;
					FileOutputStream fos = null;
					try {
						// 创建本地文件和目录
						String serverPath = "/" + descFileDir + "/" + datePath + "/" + fileName;
						String descFilePath = appRootPath + serverPath;
						File descFile = new File(descFilePath);
						File srcFile = new File(imageSrc);
						if (srcFile.exists() && (!descFile.exists() || descFile.length() <= 0)) {
							Assert.notNull(descFile, "文件不能为空");
							File parentFile = descFile.getParentFile();
							if (!parentFile.exists()) {
								parentFile.mkdirs();
							}
							if (!descFile.exists()) {
								descFile.createNewFile();
							}

							fis = new FileInputStream(srcFile);
							fos = new FileOutputStream(descFile);
							int byteRead = 0;
							byte[] buffer = new byte[8192];
							while ((byteRead = fis.read(buffer, 0, 8192)) != -1) {
								fos.write(buffer, 0, byteRead);
							}
							data = data.replace(imageSrc, serverPath);
						}
					} catch (Exception e) {
						logger.error("拷贝文件失败", e);
					} finally {
						try {
							if (fos != null) {
								fos.close();
							}
							if (fis != null) {
								fis.close();
							}
						} catch (IOException e) {
							e.printStackTrace();
						}
					}
				}
			}
		}
		return data;
	}

	// /*
	// * (non-Javadoc)
	// *
	// * @see
	// * com.collection.service.CollectionTaskService#importData(java.util.List,
	// * com.collection.entity.DBConfig)
	// */
	// @SuppressWarnings("unchecked")
	// public void importData(List<Map<String, List<String>>> dataList,
	// DBConfig dbConfig) {
	// if (dbConfig != null) {
	// DB db = dbConfig.getDb();
	// Map<DB, String> driverClassMap = dbConfig.getDriverClassMap();
	// Map<DB, String> dbURLMap = dbConfig.getDbURLMap();
	// String userName = dbConfig.getUserName();
	// String password = dbConfig.getPassword();
	// String sql = dbConfig.getImportSQL();
	//
	// DBUtil dbUtil = new DBUtil(dbURLMap.get(db),
	// driverClassMap.get(db), userName, password);
	//
	// Connection conn = dbUtil.openConnction();
	// PreparedStatement pstm = null;
	// Timestamp now = new Timestamp(System.currentTimeMillis());
	// try {
	// conn.setAutoCommit(false);
	// if (dataList != null && dataList.size() > 0) {
	// for (Map<String, List<String>> data : dataList) {
	//
	// Set<String> filedNames = data.keySet();
	//
	// Map<String, String> filedMap = new LinkedMap();
	// sql = "insert into article(";
	// String placeholder = "";
	// for (String filedName : filedNames) {
	// sql += filedName + ",";
	// placeholder += "?,";
	// List<String> collectionDatas = data.get(filedName);
	// if (collectionDatas != null
	// && collectionDatas.size() > 0) {
	// filedMap.put(filedName, collectionDatas.get(0));
	// } else {
	// filedMap.put(filedName, null);
	// }
	// }
	//
	// sql +=
	// "cat_id, click_count, display_order, hot_type, create_date, update_date) values("
	// + placeholder
	// + "10,0,0,0,'"
	// + now
	// + "','"
	// + now + "')";
	//
	// System.out.println(sql);
	// pstm = conn.prepareStatement(sql);
	// Set<String> fileds = filedMap.keySet();
	// int index = 1;
	// for (String filed : fileds) {
	// String filedValue = filedMap.get(filed);
	// System.out.println(filed + ":" + filedValue);
	// try {
	// pstm.setString(index, new String(
	// filedValue == null ? ""
	// .getBytes("GB2312")
	// : filedValue.trim().getBytes(
	// "GB2312"), "gbk"));
	// } catch (UnsupportedEncodingException e) {
	// e.printStackTrace();
	// }
	// index++;
	// }
	// pstm.executeUpdate();
	// }
	// }
	//
	// conn.commit();
	// } catch (SQLException e) {
	// try {
	// conn.rollback();
	// } catch (SQLException e1) {
	// e1.printStackTrace();
	// }
	// e.printStackTrace();
	// } finally {
	// dbUtil.close(conn, pstm, null);
	// }
	// }
	// }

	/*
	 * (non-Javadoc)
	 * 
	 * @see
	 * com.collection.service.CollectionTaskService#importPageContent(java.util
	 * .Map, java.lang.String, java.lang.String)
	 */
	@Override
	public void importPageContent(Map<String, String> contentMap, String url, String tableName) {
		StringBuffer sb = new StringBuffer("update " + tableName + " set hasCollection=1,");
		if (contentMap != null && contentMap.size() > 0) {
			Set<String> fieldNames = contentMap.keySet();
			List<String> updateParams = new ArrayList<String>();
			List<Object> updateContents = new ArrayList<Object>();
			for (String fieldName : fieldNames) {
				updateParams.add(fieldName + " = ?");
				updateContents.add(contentMap.get(fieldName));
			}

			sb.append(StringUtils.join(updateParams, ",")).append(" where pageUrl = ?")
					.append(" and hasCollection = 0");
			updateContents.add(url);
			collectionTaskDao.executeSQLByParams(sb.toString(), updateContents.toArray());
		}
	}

	public int countHasNoCollectionUrls(String tableName) {
		String sql = "select count(*) from " + tableName + " where hasCollection = 0 ";
		return collectionTaskDao.queryForIntSQL(sql);
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see
	 * com.collection.service.CollectionTaskService#findHasNoCollectionUrls(
	 * java.lang.String)
	 */
	public List<String> findHasNoCollectionUrls(String tableName, int start, int length) {
		String sql = "select pageUrl from " + tableName + " where hasCollection = 0 limit ?, ?";

		List<Map<String, Object>> dataRs = collectionTaskDao.queryForListSQL(sql, start, length);

		List<String> urls = null;
		if (dataRs != null && dataRs.size() > 0) {
			urls = new ArrayList<String>();
			for (Map<String, Object> data : dataRs) {
				for (Entry<String, Object> dataEntry : data.entrySet()) {
					urls.add((String) dataEntry.getValue());
				}
			}
		}
		return urls;
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see
	 * com.collection.service.CollectionTaskService#importPageUrl(java.util.
	 * List, java.lang.String)
	 */
	@Override
	public void importPageUrl(List<String> urls, String tableName) {
		StringBuffer sb = new StringBuffer("insert into " + tableName + "(hasCollection, hasPublish, pageUrl) values");

		String pageUrlIsExist = "select * from " + tableName + " where pageUrl = ?";
		if (urls != null && urls.size() > 0) {
			List<String> insertContents = new ArrayList<String>();
			List<Map<String, Object>> dataRs = null;
			for (String url : urls) {
				dataRs = collectionTaskDao.queryForListSQL(pageUrlIsExist, url);
				if ((dataRs == null || dataRs.size() <= 0) && StringUtils.isNotBlank(url)) {
					insertContents.add("(0, 0, '" + url + "')");
				}
			}

			if (insertContents.size() > 0) {
				sb.append(StringUtils.join(insertContents, ","));
				collectionTaskDao.executeSQL(sb.toString());
			}
		}
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see
	 * com.collection.service.CollectionTaskService#publishContent(com.collection
	 * .entity.CollectionTask)
	 */
	public List<Map<String, Object>> publishContent(PublishConfig publishConfig) {
		List<Map<String, Object>> dataMaps = null;
		if (publishConfig != null) {

			HtmlCleaner cleaner = new HtmlCleaner();

			CollectionTask task = publishConfig.getCollectionTask();

			String collectionDataCharCode = publishConfig.getCollectionDataCharCode();
			String dbCharCode = publishConfig.getDbCharCode();
			if (task != null) {
				String collectionTableName = task.getTaskAlias();

				String sql = "select count(*) from " + collectionTableName + " where hasPublish = 0";

				String updateSql = "update " + collectionTableName + " set hasPublish=1 where pageUrl=?";

				int count = collectionTaskDao.queryForIntSQL(sql);

				Date now = new Date();
				SimpleDateFormat sdf = new SimpleDateFormat();
				if (count > 0) {
					int pageSize = 100;

					int totalPage = (count * pageSize - 1) / pageSize + 1;

					List<String> filedNames = null;
					List<String> filedValueParams = null;
					List<Object> filedValues = null;

					List<CollectionModelToPublishModel> collectionModelToPublishModels = publishConfig
							.getCollectionModelToPublishModels();
					DB db = publishConfig.getDb();
					Map<DB, String> driverClassMap = publishConfig.getDriverClassMap();
					Map<DB, String> dbURLMap = publishConfig.getDbURLMap();
					String userName = publishConfig.getUserName();
					String password = publishConfig.getPassword();

					String applicationRootPath = publishConfig.getApplicationRootPath();

					String publishTableName = publishConfig.getPublishTableName();

					DBUtil dbUtil = new DBUtil(dbURLMap.get(db), driverClassMap.get(db), userName, password);

					Connection conn = dbUtil.openConnction();
					PreparedStatement pstm = null;
					try {
						conn.setAutoCommit(false);
						for (int i = 1; i <= totalPage; i++) {

							sql = "select * from " + collectionTableName
									+ " where hasCollection=1 and hasPublish = 0 limit " + (i - 1) * pageSize + ","
									+ pageSize;

							dataMaps = collectionTaskDao.queryForListSQL(sql);

							for (Map<String, Object> dataMap : dataMaps) {
								filedNames = new ArrayList<String>();
								filedValueParams = new ArrayList<String>();
								filedValues = new ArrayList<Object>();
								for (CollectionModelToPublishModel ctp : collectionModelToPublishModels) {

									ModelType modelType = ctp.getModelType();
									DataType dataType = ctp.getDataType();

									String fieldName = ctp.getFieldName();

									String fieldValue = ctp.getFieldValue();

									String filePath = ctp.getFilePath();

									int splitNums = ctp.getSplitNums();

									filedNames.add(fieldName);
									Object value = null;
									if (modelType != null && modelType.equals(ModelType.EXPRESSION)) {
										value = dataMap.get(fieldValue);
										if (splitNums > 0 && value != null) {

											TagNode tagNode = cleaner.clean((String) value);
											if (tagNode != null) {
												/*
												 * 截取指定长度的字符串
												 */
												StringBuffer sb = tagNode.getText();
												if (sb != null) {
													String content = sb.toString().trim();
													if (content.length() > splitNums) {
														value = content.substring(0, splitNums);
													} else {
														value = content;
													}
												}
											}
										}
									} else {
										value = fieldValue;
									}

									/*
									 * 如果文件路径不为空说明有下载文件需要拷贝
									 */
									if (StringUtils.isNotBlank(filePath)) {
										value = copySRCFileToDescFile((String) value, applicationRootPath, filePath);
										System.out.println(value);
									}

									/*
									 * 根据数据类型不同，插入SQL语句形式有所不同
									 */
									Object fieldValueObj = value;
									if (dataType != null) {
										if (dataType.equals(DataType.DATE)) {
											sdf.applyPattern((String) value);
											fieldValueObj = sdf.format(now);
										} else if (dataType.equals(DataType.NUMBER)) {
											fieldValueObj = Integer.parseInt((String) value);
										}
									}

									if (StringUtils.isNotBlank(collectionDataCharCode)
											&& StringUtils.isNotBlank(dbCharCode)
											&& !collectionDataCharCode.equals(dbCharCode)
											&& (fieldValueObj != null && fieldValueObj instanceof String)) {
										String fieldValueStr = (String) fieldValueObj;
										System.out.println(collectionDataCharCode + "---" + dbCharCode);
										fieldValueObj = new String(fieldValueStr.getBytes(collectionDataCharCode),
												dbCharCode);
									}
									filedValueParams.add("?");
									filedValues.add(fieldValueObj);
								}
								sql = "insert into " + publishTableName + "(" + StringUtils.join(filedNames, ",")
										+ ") values(" + StringUtils.join(filedValueParams, ",") + ")";
								System.out.println(sql);

								if (filedValues != null && filedValues.size() > 0) {
									pstm = conn.prepareStatement(sql);

									for (int v = 0; v < filedValues.size(); v++) {
										Object filedValue = filedValues.get(v);
										if (filedValue != null && (filedValue instanceof String)) {
											/*
											 * 避免字符串中含有乱码，无法插入数据库
											 */
											pstm.setBytes(v + 1, ((String) filedValue).getBytes());
										} else {
											pstm.setString(v + 1, filedValue + "");
										}
									}
									pstm.executeUpdate();
									/*
									 * 将采集的数据的发布状态更新为已发布
									 */
									collectionTaskDao.executeSQLByParams(updateSql, dataMap.get("pageUrl"));
									pstm.close();
									pstm = null;
								}
							}
						}
						conn.commit();
					} catch (SQLException e) {
						try {
							conn.rollback();
						} catch (SQLException e1) {
							e1.printStackTrace();
						}
						e.printStackTrace();
					} catch (UnsupportedEncodingException e) {
						e.printStackTrace();
					} finally {
						dbUtil.close(conn, pstm, null);
					}
				}
			}
		}
		return dataMaps;
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see
	 * com.collection.service.CollectionTaskService#createUrl(java.util.List,
	 * java.lang.String, int, java.util.List)
	 */
	public void createUrl(List<UrlParam> urlParams, String url, int i, List<String> urls) {
		List<String> tempUrls = new ArrayList<String>();
		int size = urlParams.size();
		UrlParam urlParam = urlParams.get(i);
		Integer start = urlParam.getStart();
		Integer end = urlParam.getEnd();
		Integer step = urlParam.getStep();
		do {
			String tempUrl = StringUtils.replace(url, "{*}", start.toString(), 1);
			tempUrls.add(tempUrl);
			start += step;
		} while (start <= end);

		if (size > 1 && i < size - 1) {
			i++;
			for (String tempUrl : tempUrls) {
				createUrl(urlParams, tempUrl, i, urls);
			}
		} else if (i == size - 1) {
			for (String tempUrl : tempUrls) {
				urls.add(tempUrl);
			}
			return;
		}
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see
	 * com.collection.service.CollectionTaskService#getHtmlContent(java.lang
	 * .String, java.lang.String)
	 */
	public String getHtmlContent(String path, String charCode) {
		URL url = null;
		BufferedReader br = null;
		String str = "";
		StringBuffer sb = new StringBuffer();
		try {
			url = new URL(path);
			// System.out.println(url.getAuthority() + ",\n" +
			// url.getDefaultPort() + ",\n" + url.getFile() + ",\n" +
			// url.getHost() + ",\n"
			// +url.getPath() + ",\n" + url.getPort() + ",\n" +url.getQuery() +
			// ",\n"+url.getRef() + ",\n"+url.getUserInfo() +
			// ",\n"+url.getContent());

			br = new BufferedReader(new InputStreamReader(url.openStream(), charCode));

			while ((str = br.readLine()) != null) {
				sb.append(str.trim());
			}

		} catch (MalformedURLException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		} finally {
			try {
				if (br != null) {
					br.close();
				}
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
		return sb.toString();
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see
	 * com.collection.service.CollectionTaskService#getHost(java.lang.String)
	 */
	public String getHost(String path) {
		try {
			URL url = new URL(path);
			System.out.println(url.getAuthority() + ",\n" + url.getDefaultPort() + ",\n" + url.getFile() + ",\n"
					+ url.getHost() + ",\n" + url.getPath() + ",\n" + url.getPort() + ",\n" + url.getQuery() + ",\n"
					+ url.getRef() + ",\n" + url.getUserInfo());

			return url.getHost();
		} catch (MalformedURLException e) {
			e.printStackTrace();
		}
		return null;
	}

	public void setTableName(String tableName) {
		this.tableName = tableName;
	}

	public void setTest(boolean isTest) {
		this.isTest = isTest;
	}
}
